package com.foreveross.crawl.adapter.sub.impl20140402.v3;

import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import net.sf.json.JSONObject;

import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.tools.ant.taskdefs.Sleep;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.foreveross.crawl.adapter.AbstractAdapter;
import com.foreveross.crawl.adapter.CrawlAdapterFactory;
import com.foreveross.crawl.adapter.PlaneInfoEntityBuilder;
import com.foreveross.crawl.common.util.DateUtil;
import com.foreveross.crawl.common.util.RegHtmlUtil;
import com.foreveross.crawl.domain.airfreight.AgentEntity;
import com.foreveross.crawl.domain.airfreight.CabinEntity;
import com.foreveross.crawl.domain.airfreight.TransitEntity;
import com.foreveross.crawl.domain.airfreight.doub.DoublePlaneInfoEntity;
import com.foreveross.crawl.domain.airfreight.doub.ReturnAgentEntity;
import com.foreveross.crawl.domain.airfreight.doub.ReturnCabinEntity;
import com.foreveross.crawl.domain.airfreight.doub.ReturnDoublePlaneInfoEntity;
import com.foreveross.crawl.domain.airfreight.doub.ReturnTransitEntity;
import com.foreveross.crawl.domain.airfreight.single.SinglePlaneInfoEntity;
import com.foreveross.crawl.exception.FlightInfoNotFoundException;
import com.foreveross.taskservice.common.bean.TaskModel;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

public class KayakAdapter extends AbstractAdapter{
	/**
	 * 航班正则的分割符号
	 */
//	private String REGEX_FLIGHTNO="Flight";
	private String REGEX_FLIGHTNO="航班";
	
	public static final int KAYAK_NEXPAGE_REPEATCOUNT=10;
	public static final int KAYAK_DETAIL_REPEATCOUNT=3;
	public static final String KAYAK_NEED_CARRIERKEY="CA|CZ|HU|SC|MU|ZH|MF|3U";
	public static final int KAYAK_PAGE_PAGELEN=60;
	
	private static Integer backCount=2;
	private static Integer backCount2=2;
	private static Integer backCount3=2;
	private static Integer out_time=1;
	private static final int PAGE_FINAL=15;
	
	private int LOCL_PAGE_INDEX=0;//当前页码
	private int LOCL_PAGE_INDEX_FLAG=1;//当前页码
	private int PAGE_OR_NOT=0;//当前页码
	
	public static final boolean FILTER_CONPANY=true;
	
	
	public static final Map<String,String> flightS=Maps.newHashMap();
	{
		flightS.put("PEK-SEA", "HU,UA,DL");
		flightS.put("PEK-ORD", "HU,UA,AA");
		flightS.put("PEK-BOS", "HU,UA,AA,DL");
		flightS.put("PEK-YYZ", "HU,AC,DL");
		flightS.put("PEK-HNL", "CA,MU,HA,OZ,KE,NH,JL");
		flightS.put("PEK-IAD", "CA,UA,KE,NH");
		flightS.put("PEK-IAH", "CA,UA,AA");
		flightS.put("PEK-JFK", "CA,MU,AA,UA,DL,OZ,KE,NH,JL,CX");
		flightS.put("PEK-LAX", "CA,CZ,MU,AA,UA,DL,OZ,KE,NH,JL,CX");
		flightS.put("PEK-SFO", "CA,MU,UA,DL,OZ,KE,NH");
		flightS.put("PEK-YVR", "CA,MU,AC");
		flightS.put("PEK-ARN", "CA");
		flightS.put("PEK-SVO", "CA,HU,SU");
		flightS.put("PEK-MUC", "CA,CZ,EK,SU,LH");
		flightS.put("PEK-FRA", "CA,MU,CZ,EK,SU,LH");
		flightS.put("PEK-CDG", "CA,MU,CZ,EK,SU,AF");
		flightS.put("PVG-FRA", "CA,MU,EK,SU,LH");
		flightS.put("PVG-CDG", "CA,MU,EK,SU,AF");
		flightS.put("PEK-HKG", "CA,CZ,HU,HX,CX");
		flightS.put("PEK-LHR","CA,MU,CZ,EK,SU");
	};
	//public List<String> totalCompany=Lists.newArrayList();
	public String filterPageCompany=""; //从页面获取所得到的过滤条件
	
	public static final StringBuffer filterCompany=new StringBuffer();
	{
		filterCompany.append("VN,");
		filterCompany.append("LX,");
		filterCompany.append("ZH,");
		filterCompany.append("MH,");
		filterCompany.append("MF,");
		filterCompany.append("QR,");
		filterCompany.append("QF,");
		filterCompany.append("UN,");
		filterCompany.append("AZ,");
		filterCompany.append("AY,");
		filterCompany.append("UL,");
		filterCompany.append("EY,");
		filterCompany.append("MS,");
		filterCompany.append("AB,");
		filterCompany.append("ET,");
		filterCompany.append("AI,");
		filterCompany.append("TG,");
		filterCompany.append("TK,");
		filterCompany.append("BR,");
		filterCompany.append("GA,");
		filterCompany.append("JJ,");
		filterCompany.append("OM,");
		filterCompany.append("KA,");
		filterCompany.append("KC,");
		filterCompany.append("KL,");
		filterCompany.append("S7,");
		filterCompany.append("SA,");
		filterCompany.append("3U,");
		filterCompany.append("SK,");
		filterCompany.append("OS,");
		filterCompany.append("SQ,");
		filterCompany.append("MULT");
	};
	
	//SU,AC,CA,NH,CX,CZ,KA,ET,BR,AY,GA,HX,JL,KE,MH,OM,QR,ZH,SQ,SA,TG,TK,VN,MF,MULT,DL
	public String getREGEX_FLIGHTNO() {
		return REGEX_FLIGHTNO;
	}

	public void setREGEX_FLIGHTNO(String rEGEX_FLIGHTNO) {
		REGEX_FLIGHTNO = rEGEX_FLIGHTNO;
	}

	public KayakAdapter(TaskModel taskQueue) {
		super(taskQueue);
	}
	
	private static Map<String,Integer> dateMap = new HashMap<String, Integer>();
	{
		//英文
		/*dateMap.put("Mon", 1);
		dateMap.put("Tue", 2);
		dateMap.put("Wed", 3);
		dateMap.put("Thr", 4);
		dateMap.put("Thu", 4);
		dateMap.put("Fri", 5);
		dateMap.put("Sat", 6);
		dateMap.put("Sun", 7);*/
		
		//中文
		dateMap.put("星期一", 1);
		dateMap.put("星期二", 2);
		dateMap.put("星期三", 3);
		dateMap.put("星期四", 4);
		dateMap.put("星期五", 5);
		dateMap.put("星期六", 6);
		dateMap.put("星期日", 7);
	}
	
	
	@Override
 	public Object fetch(String url) throws Exception {
		List<String> resultList=Lists.newArrayList();
		//重写此步骤（NEW）
		HttpClient httpClient = this.getHttpClient();
		HttpGet g = new HttpGet(firstR());
		setRequestHeadInfo(g);
		String page1=null;
		String seacheId=null;
		HttpPost httpPost = null;
		try{
			logger.info("获取第一层数据，并解析此次的searchId...");
			page1= excuteRequest(httpClient, g, "gb2312");
			if(page1 ==null){
				logger.error("第一层数据为空，回滚...");
				while(backCount-->0)
					resultList= (List<String>) fetch(null);
			}else{//获取此次查询的seacheID
				logger.info("开始解析seachId...");
				seacheId=analyzeId(page1);
				if(seacheId !=null){
					logger.info(String.format("解析searchId=[%s];状态：成功", seacheId));
					List<String> resultLists=Lists.newArrayList();
					logger.info("进入并解析第一页...");
					resultLists=dataListTotal(seacheId,0,httpClient,httpPost);//第一页
					if(resultLists.size()>0)
						resultList.addAll(resultLists);
					if(LOCL_PAGE_INDEX >0){//从第二页开始
						for(int k=1;k<LOCL_PAGE_INDEX;k++){
							resultLists=dataListTotal(seacheId,k,httpClient,httpPost);//第一页
							if(resultLists.size()>0)
								resultList.addAll(resultLists);
						}
					}
				}else{
					while(backCount2-->0){
						logger.error(String.format("解析searchId=[%s];状态：失败;回滚执行一次...", seacheId));
						resultList= (List<String>) fetch(null);
					}
					/*
					logger.info("切换IP，重新从首页进入...");
					switchProxyipByWebClient();
					*/
					//;
					//throw new Exception("出现验证码!");
				}
				
			}
		}catch (Exception e) {
				while(backCount3-->1){
					logger.error("操作异常，回滚执行一次");
					resultList= (List<String>) fetch(null);
				}
//				throw new Exception("回滚一次执行失败,原因：可能出现验证码，请检查!");
				logger.error("回滚一次执行失败!");
		}
		
		if(resultList.size()>0){
			int lenght=0;
			for(String s:resultList){
				super.appendPageContents(s);
				lenght+=s.length();
			}
			setLenghtCount(lenght);
		}
		
		return resultList;
		//（old）
		/*WebClient w = null;
		try {
			w=getWebClient();
			logger.info(String.format("获得浏览器版本[%s]", w.getBrowserVersion()));
			return fetchFilterPage(w,0,1,null);
		}finally{
//			closeWebClient(w);
			destory();
		}*/
	}
	
	
	public List<String> dataListTotal(String searchId,int pageIndex,HttpClient httpClient,HttpPost httpPost){
		List<String> resultList=Lists.newArrayList();
		String url2=secondR(searchId,pageIndex);//过滤地址
		//PAGE_OR_NOT++;
		logger.info(String.format("点击第%s页，获取过滤后的数据,并解析相应数据...",pageIndex+1));
		String result2=null;
		
		try{
			result2=getPageByUrl(url2,httpClient,httpPost);
			//获取对应的index resulid && 总页数
			if((result2==null || result2.contains("flightresult resultrow")) && filterPageCompany.equals("")){
				
				//*******************新的过滤条件**************************************
				getAllCompany(result2);//获取所有的航空公司
				PAGE_OR_NOT++;
				url2=secondR(searchId,pageIndex);//重新组装地址URL
				result2=null;
				result2=getPageByUrl(url2,httpClient,httpPost);//重新查找数据
				//****************************************************************
			}
			if(result2==null || result2.contains("flightresult resultrow")){//System.out.println(result2);
				Map<String ,String> resultMap=Maps.newHashMap();
				resultMap=indexData(result2);
				logger.info(String.format("当前共有%s页", LOCL_PAGE_INDEX+1));
				if(resultMap.size()>0){
					int j=0;
					for(Map.Entry<String, String>indexMap:resultMap.entrySet()){
						j++;
						logger.info(String.format("解析当前第%s/%s页，点击详情【%s/%s】", pageIndex+1,LOCL_PAGE_INDEX,j,resultMap.size()));
						Thread.sleep(500);
						String urlF=thirdR(searchId,indexMap.getKey(),indexMap.getValue());//String seachid,String resultid,String localidx
						String detailStr=getPageByUrl(urlF,httpClient,httpPost);
						/*if(detailStr !=null){
							detailStr="<html><head></head><body>"+detailStr+"</body></html>";
						}*/
						
						JSONObject obj=JSONObject.fromObject(detailStr);
						if(obj.getInt("status")==0){
							detailStr=obj.getString("message");
							if(detailStr !=null){
								logger.info(String.format("点击详情[%s/%s]成功",j,resultMap.size()));
								resultList.add(detailStr);

							}
						}else if(obj.getString("message").contains("已过期")&&obj.getInt("status")==-1){
							logger.error("获取的详情页出错：操作过期，重新执行一次");
							while(out_time-->0)
								resultList=dataListTotal(searchId,pageIndex,httpClient,httpPost);
						}
						
					}
				
				}
			}	
		}catch (Exception e) {
			logger.error("获取第二层数据出错！");
			return resultList.size()>0?resultList:null;
		}
		return resultList;
	}
	
	
	
	/**
	 * 第一层url
	*/
	//http://www.kayak.com/flights/PEK-HKG/2014-08-16/2014-08-23
	public String firstR(){
		StringBuffer url=new StringBuffer("http://www.cn.kayak.com/flights/");
		url.append(super.taskQueue.getFromCity()+"-");
		url.append(super.taskQueue.getToCity()+"/");
		url.append(super.taskQueue.getFlightDate()+"/");
		url.append(super.taskQueue.getReturnGrabDate());
		return url.toString();
	}
	
	/**
	 * 第二层URL
	 * 带过滤条件
	*/
	public String secondR(String seachId,int pageIndex){
		StringBuffer url=new StringBuffer();
		if(PAGE_OR_NOT==1){
			url.append("http://www.cn.kayak.com/s/jsfilter?");
		}else{
			url.append("http://www.cn.kayak.com/s/jspage?");
		}
			
		url.append("ss=1");
		url.append("&c=15");
//		StringBuffer url=new StringBuffer();
		String fs=null;//过滤条件
		//支撑航信数据，现在过滤掉有2次以上的上转信息（只查1次或者不中转的信息）
		if(FILTER_CONPANY){//过滤航空公司
			fs="stops=2;";
			if(filterPageCompany !=null && !filterPageCompany.toString().equals("")){
				fs+="airlines="+filterPageCompany+";";
			}/*else{
				fs+="airlines="+filterCompany.toString()+";";
			}*/
			fs+="codeshare=true";
		}
		logger.info(String.format("过滤规则[%s]", fs));
		url.append("&fs="+URLEncoder.encode(fs));
		url.append("&itd=1");
		url.append("&lmname=");
		url.append("&lmname2=");
		//url.append("&pn="+pageIndex);
		url.append("&poll=-1");
		url.append("&s=price");
		url.append("&searchid="+seachId);
		url.append("&seo=false");
		url.append("&streaming=false");
		//url.append("&ua=FLT");
		url.append("&urlViewState=");
		String vs="{\"pn\":"+pageIndex+",\"fodi\":\"RESET\",\"codi\":\"RESET\",\"ws\":}";
		url.append("&viewState="+URLEncoder.encode(vs));
		url.append("&vw=list");
		return url.toString();
	}
	
	/**
	 * 第三层URL(详情页URL)
	*/
	public String thirdR(String seachid,String resultid,String localidx){
		String result=null;
		StringBuffer url=new StringBuffer("http://www.cn.kayak.com/s/run/inlineDetails/flight?");
		String fs=null;//过滤条件
		//支撑航信数据，现在过滤掉有2次以上的上转信息（只查1次或者不中转的信息）
		if(FILTER_CONPANY){//过滤航空公司
			fs="stops=2;";
			if(filterPageCompany !=null && !filterPageCompany.toString().equals("")){
				fs+="airlines="+filterPageCompany+";";
			}/*else{
				fs+="airlines="+filterCompany.toString()+";";
			}*/
			fs+="codeshare=true";
		}
		logger.info(String.format("过滤规则[%s]", fs));
		url.append("&fs="+URLEncoder.encode(fs));
		url.append("&localidx="+localidx);
		url.append("&resultid="+resultid);
		url.append("&searchid="+seachid);
		url.append("&suppressLookback=false");
		
		return url.toString();
	}
	
	
	/**
	 * 根据URL获取Page
	*/
	public String getPageByUrl(String Url,HttpClient httpClient, HttpPost httpPost){
		String result=null;
		try {
			if(Url !=null && httpClient !=null){
				Thread.sleep(1000);
				httpPost = new HttpPost(Url);
				setPostHeadInfo(httpPost);//设置头部信息
				Thread.sleep(1000);
				result=excuteRequest(httpClient, httpPost, "UTF-8");
			}
			
			if(result !=null&&result.contains(".*搜索可能已过期.*")){
				throw new Exception("操作过期！");
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		return result;
	}
	
	
	/**
	*/
	public void getAllCompany(String page){
		if(page.contains("for=\"fs_airlines")){
			//获取当前所有的航空公司
			List<String> companyC=Lists.newArrayList();
			companyC=getPatternList(page, "for=\"fs_airlines_([A-Z]{2,4})_input\"", " ", "");//System.out.println(page);
			if(companyC.size()>0 && filterPageCompany.equals("")){
				String flightValue=null;
				flightValue=flightS.get(taskQueue.getFromCity()+"-"+taskQueue.getToCity());
				if(!flightValue.equals("") && flightValue !=null){
					for(int i=0;i<companyC.size();i++){
						String key=companyC.get(i);
						for(String keyT:flightValue.split(",")){
							if(keyT.equals(key)){
								companyC.remove(i);
								i--;
								break;
							}
						}
					}
				}
				for(String keyG:companyC){
					filterPageCompany+=keyG+",";
				}
				filterPageCompany =filterPageCompany.substring(0, filterPageCompany.length()-1);
			}
			
			
		}else{
			logger.error("获取所有的航空公司出错!");
		}
		
	}
	
	
	/**
	 * 设置请求头信息，通用
	 * 
	 * @param g
	 */
	private void setRequestHeadInfo(HttpRequestBase g) {
		g.setHeader("Host", "www.cn.kayak.com");
		g.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; rv:31.0) Gecko/20100101 Firefox/31.0");
		g.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
//		g.setHeader("Accept-Encoding", "gzip, deflate");
		g.setHeader("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
	}
	
	private void setPostHeadInfo(HttpPost httpPost) {
		httpPost.setHeader("Host", "www.cn.kayak.com");
		httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; rv:31.0) Gecko/20100101 Firefox/31.0");
		httpPost.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
//		httpPost.setHeader("Accept-Encoding", "gzip, deflate");
		httpPost.setHeader("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
		httpPost.setHeader("Referer",firstR());
	}
	
	//第一层解析seachId
	public String analyzeId(String page){
		String seachId=null;
		if(page.contains("originsid") ){//System.out.println(page);
			String reg="(<input\\s*?type=\"hidden\"\\s*?name=\"originsid\"\\s*?value=\"(.*)\"\\s*?/>)";
			String inputH=null;
			inputH=RegHtmlUtil.regStr(page,reg);
			seachId=RegHtmlUtil.regStr(inputH,"(value=\"(.*)\")").split("\"")[1].replaceAll("\"", "");
		}
		
		return seachId;
	}
	
	/**
	 * 解析第二层
	*/
	public Map<String,String> indexData(String result){
		Map<String,String> dataList=Maps.newHashMap();
		String key="class=\"flightresult\\s*resultrow(.*?)>";
		String pageKey="\\d*</a></span><span\\s*?class=\'pglink\\s*?\'>";//8</a></span><span  class='pglink '>
		
		try{
		List<String> info=Lists.newArrayList();
		List<String> pageL=Lists.newArrayList();
		String pageStr=null;
			//if (LOCL_PAGE_INDEX < 1) {// 12</a></span><spanclass='pglink'>
				pageL=getPatternList(result, "(\\d*</a></span><span\\s*?class=\'pglink\\s*?\'>)"," ","");
				if (pageL.size() > 0) {
					pageStr = pageL.get(pageL.size() - 1);
					if (pageStr != null&& RegHtmlUtil.regStr(pageStr, "([0-9]+)").matches("[0-9]+")) {// 获取当前最大页码
						LOCL_PAGE_INDEX = Integer.parseInt(RegHtmlUtil.regStr(pageStr, "([0-9]+)"));
					}
				}
		//	}
		info=getPatternList(result,key," ","");
		if(info.size()>0){
			String reId="data-resultid=\"([a-zA-Z0-9]*)";
			String index="data-index=\"([a-zA-Z0-9]*)";
			String key1="",key2="";
			for(String str:info){
				key1=RegHtmlUtil.regStr(str, reId);
				key2=RegHtmlUtil.regStr(str, index);
				if(!key1.equals("") && key1 !=null && !key2.equals("") && key2 !=null){
					dataList.put(key1, key2);
				}
			}
		}else
			logger.error("无法获取密钥航段对应的index");
		}catch (Exception e) {
			logger.error("第二层解析失败");
			e.printStackTrace();
		}
		return dataList;
	}
	
	private List<String> getPatternList(String content, String regEx, String replace, String replacement){
		Pattern p=Pattern.compile(regEx, Pattern.DOTALL|Pattern.MULTILINE);
		Matcher m=p.matcher(content);
		List<String> strList = new ArrayList<String>();
		while(m.find()){
			strList.add(m.group(1).trim().replaceAll(replace, replacement));
		}
		return strList;
	}
	
	/*public static List<String> regStrs(String sHtml, String mode) {
		Pattern p = Pattern.compile(mode, Pattern.CASE_INSENSITIVE + Pattern.DOTALL + Pattern.MULTILINE);
		List<String> list = Lists.newArrayList();
		Matcher m = p.matcher(sHtml);
		if (m.find()) {
			for (int i = 0; i <= m.groupCount(); i++) {
				list.add(m.group(i));
			}
		}
		return list;
	}*/
	
	private List<String>  fetchFilterPage(WebClient w,int pageIndex,int repeatCount,List<String> result) throws Exception{
		if(result==null || result.size()<0){
			result=new ArrayList<String>();
		}
		HtmlPage htmlPage = null;
		String indexHtml;
		String searchid = null;
		String fs = null;
		while(repeatCount-->0){
			try {
				htmlPage=fetchHomeProcess(w);
				indexHtml = htmlPage.asXml();
				validateEachPage(indexHtml, true);
				searchid =RegHtmlUtil.regStr(indexHtml, "SearchID\\s*?=\\s*?\"(.*?)\"");
				if(searchid==null){
					throw new Exception("无法获取密钥:searchid");
				}
//				Document doc=Jsoup.parse(indexHtml);
//				Elements chb=doc.select("input[name=airlines]");

				
				break;
			} catch (Exception e) {
				if(e instanceof FlightInfoNotFoundException){
					throw e;
				}
				logger.warn(String.format("异常:\n[%s]", e));
				if(repeatCount>0){//重新走首页
					logger.info("切换IP，重新从首页进入...");
					switchProxyipByWebClient();
//					changeIp(w);
				}else{
					throw e;
				}
			}
		}
//		fs=filterStr(chb);//过滤规则
		//支撑航信数据，现在过滤掉有2次以上的上转信息（只查1次或者不中转的信息）
		fs="stops=2;";
		if(FILTER_CONPANY){//过滤航空公司
			fs+="airlines="+filterCompany.toString();
		}
		fs+=";codeshare=true";
		logger.info("过滤规则:"+fs);
		int pageLen=KAYAK_PAGE_PAGELEN;
		Map<String,String> map=new HashMap<String,String>();
		map.put("c", pageLen+"");
		map.put("fs", fs);
		map.put("itd", "1");
		map.put("lmname", "");
		map.put("lmname2", "");
		map.put("poll", "-1");
		map.put("s", "price");
		map.put("searchid", searchid);
		map.put("seo", "false");
		map.put("streaming", "false");
		map.put("urlViewState", "");
		map.put("vw", "list");
		map.put("ss", "1");
		while(true){
			map.put("viewState", "{\"pn\":"+pageIndex+",\"fodi\":\"RESET\",\"codi\":\"RESET\",\"ws\":\"\"}");
			try {
				String filterUrl=getBaseGetUrl("http://www.cn.kayak.com/s/jsresults?", map);
				Page nxPage=w.getPage(filterUrl);
				String nxHtml ="<html><head></head><body>"+nxPage.getWebResponse().getContentAsString()+"</body></html>";
				validateEachPage(nxHtml,true);
				logger.info(String.format("进入页码[%s]", pageIndex));
				int size=fetchDetailInfo(w,nxHtml, result, searchid);
				if(size<pageLen){
					break;
				}
			} catch (Exception e) {	
				logger.warn(String.format("进入[%s]页失败 异常:\n[%s]", pageIndex,e));
				int nextPageRepeat=KAYAK_NEXPAGE_REPEATCOUNT;
				return fetchFilterPage(w,pageIndex,nextPageRepeat,result);
			}
			pageIndex++;
		}
		return result;
	}
	
	private HtmlPage fetchHomeProcess(WebClient w)throws Exception{
		HtmlPage page=w.getPage("http://www.cn.kayak.com/flights");
		logger.info("进入官网");
//		HtmlPage  htmlPage=w.getPage("http://www.kayak.com.hk/in?url=/&cc=hk&a=kayak&p=&regen=1&lc=zh");
//		logger.info("设置语言");
		
//		List<NameValuePair> params=new ArrayList<NameValuePair>();
//		WebRequest reqSetLag=new WebRequest(new URL("http://www.kayak.com/k/run/currencysetter/setcurrency"),HttpMethod.POST);
//		params.add(new NameValuePair("newcurrency", "CNY"));
//		reqSetLag.setRequestParameters(params);
//		w.getPage(reqSetLag);
//		logger.info("请求RMB");
		
	
		
//		WebRequest reqSureLag=new WebRequest(new URL("http://www.kayak.com/k/ident/currency?currency=CNY&posted=true&save=false"),HttpMethod.POST);
//		params.clear();
//		params.add(new NameValuePair("currency", "CNY"));
//		params.add(new NameValuePair("posted", "true"));
//		params.add(new NameValuePair("save", "false"));
//		reqSureLag.setRequestParameters(params);
//		w.getPage(reqSureLag);
//		logger.info("设置RMB");
		
		
		String link="http://www.cn.kayak.com/flights/"+taskQueue.getFromCity()+"-"+taskQueue.getToCity()+"/"+
				taskQueue.getFlightDate()+(taskQueue.getIsReturn()==0?"":"/"+taskQueue.getReturnGrabDate());
		String resultLink=link+"?force=remaining&clienttimeout=true&streamerr=65000+timeout+expired";
		page=w.getPage(resultLink);
		return page;
	}
	
	//获取详情页面
	private int fetchDetailInfo(WebClient w,String html,List<String> result,String searchId) throws Exception{
		Document doc=Jsoup.parse(html);
		Elements divs=doc.select("div.flightresult");
		int size=divs.size();//divs.size()>10?divs.size()/2:divs.size();
		logger.info(String.format("本页航班数量[%s]", size));
		
		for(int i=0;i<size;i++){
			Element div=divs.get(i);
			int repeatCount=KAYAK_DETAIL_REPEATCOUNT;
			while(repeatCount-->0){
				try{
					Thread.sleep(1000);
					String localidx=RegHtmlUtil.regStr(div.html(), "flightdetail\\((.*?),");
					String resultid=RegHtmlUtil.regStr(div.html(), "flightdetail.*?'(.*?)',");
					String url="http://www.cn.kayak.com/s/run/inlineDetails/flight?" +
							"fs=codeshare=true" +
							"&localidx="+localidx+
							"&resultid="+resultid+"&searchid="+searchId+"&suppressLookback=false";
					Page p=w.getPage(url);
					JSONObject j=JSONObject.fromObject(p.getWebResponse().getContentAsString());
					if(j.getInt("status")==0){
						result.add(div.html()+j.getString("message"));
						logger.info(String.format("点击详情[%s/%s]成功", i+1,size));
						break;
					}
				}catch(Exception e){
					logger.warn(String.format("点击详情[%s/%s]异常:\n[%s]", (i+1),size,e));
//					changeIp(w);
				}
			}
		}
		return size;
	}
		
	
	/*private boolean changeIp(WebClient w){
		ProxyIpModel ip=getNewIp();
		if(ip==null){
			return false;
		}else{
			w.getOptions().setProxyConfig(new ProxyConfig(ip.getIp(), ip.getPort()));
			return true;
		}
	}*/
	private boolean validateEachPage(String html,boolean isThrowException)throws Exception{
		try{
			Document doc=Jsoup.parse(html);
			String str=doc.html().replaceAll("\\s", "");
			//System.out.println(str);
			if(str.matches(".*我們搜尋的航空公司中找不到結果.*") || str.matches(".*Noresultswerefoundamongtheairlineswesearch.*")){
				throw new FlightInfoNotFoundException();
			}
			if(isThrowException && doc.select("div.flightresult").isEmpty() ){
				throw new Exception("没有结果集");
			}
			return !doc.select("div.flightresult").isEmpty();
		}catch(Exception e){
			if(isThrowException){
				throw e;
			}
			return false;
		}
	}
	//根据航空公司配置和信息提取过滤字符串
	private String filterStr(Elements carrierKeyCheckbox){
		StringBuffer bf=new StringBuffer();
		List<String> keys=null;
		String config=KAYAK_NEED_CARRIERKEY;
		if(config!=null){
			keys=Arrays.asList(config.split("\\|"));
		}
		if(keys!=null && carrierKeyCheckbox!=null && !carrierKeyCheckbox.isEmpty()){
			for(Element chb:carrierKeyCheckbox){
				if(!keys.contains(chb.attr("value"))){
					bf.append(chb.attr("value")+",");//添加过滤
				}
			}
		}
		if("".equals(bf)){
			return "codeshare=true";
		}else{
			return "airlines="+bf.toString().replaceAll(",$", "")+";codeshare=true";
		}
	}
	
	
	@Override
	public List<Object> paraseToVo(Object fetchObject) throws Exception {
		if(taskQueue.getIsReturn()==0){
			//单程
			return paraseoOneWay(fetchObject);
		}else{
			//往返
			return paraseoRound(fetchObject);
		}
	}

	//单程
	@SuppressWarnings("unchecked")
	public List<Object> paraseoOneWay(Object fetchObject) throws Exception {
		List<String> htmls=(List<String>)fetchObject;
		List<SinglePlaneInfoEntity> entitys=new ArrayList<SinglePlaneInfoEntity>();
		for(String html:htmls){
			Document doc=Jsoup.parse("<html><body>"+html+"</body></html>");
			String airName=RegHtmlUtil.regStr(doc.select("tr.first").first().text(), "(.*?)–").replaceAll("\\s", "");
			String flightNo=getCarrierKey(airName)+RegHtmlUtil.regStr(doc.select("tr.first").first().text(), getREGEX_FLIGHTNO()+".*?(\\d.*?)\\s").replaceAll("\\s", "");
			String start=RegHtmlUtil.regStr(doc.select("td.time").first().text(),"(\\d{1,2}:\\d{1,2})").replaceAll("\\s", "");
			String startDate=doc.select("td.date").first().text();
			start=this.tranDateTime(null,start,startDate);
			start=start.length()==4?"0"+start:start;
			String end=RegHtmlUtil.regStr(doc.select("td.time").last().text(),"(\\d{1,2}:\\d{1,2})").replaceAll("\\s", "");
			String endDate=doc.select("td.date").last().text();
			end=this.tranDateTime(null,end,endDate);
			end=end.length()==4?"0"+end:end;
			SinglePlaneInfoEntity entity=PlaneInfoEntityBuilder.buildPlaneInfo(
					taskQueue, getCarrierKey(airName), airName, airName, start, end, flightNo, null, null, null, null, SinglePlaneInfoEntity.class);
			for(Element div:doc.select("div.fareInformation")){//预订网站
				String cabinName=div.attr("data-cabin");
				for(Element tb:div.select("tbody")){
					String agentName=tb.select("img").first().attr("title");
					String cabinCode=tb.select("td.fareCodes").text();
					//String priceStr=tb.select("td.total").text().replaceAll("\\D", "");
					String priceStr=getPrice(tb.select("td.total").text());
					logger.info(String.format("[%s]解析为[%s]", tb.select("td.total").text(),priceStr));
					if(NumberUtils.isNumber(priceStr)){
						Double price=Double.valueOf(priceStr);
						CabinEntity cabinEntity=PlaneInfoEntityBuilder.buildCabinInfo(
								getCabinType(cabinName), cabinCode, null, priceStr, null, null, null, CabinEntity.class);
						//税收信息
						cabinEntity.setTaxesPrice(0.0);
						cabinEntity.setOriginalPrice(cabinEntity.getPrice());
						
						entity.getCabins().add(cabinEntity);
						AgentEntity agent=new AgentEntity();
						agent.setName(agentName);
						agent.setPrice(price);
						entity.getAgents().add(agent);
						if(entity.getTotalLowestPrice()==null ||entity.getTotalLowestPrice()>price){
							entity.setTotalLowestPrice(price);
							entity.setAgentName(agentName);
						}
						if(entity.getTotalHighestPrice()==null || price>entity.getTotalHighestPrice()){
							entity.setTotalHighestPrice(price);
						}
						entity.setSumHighestPrice(entity.getTotalHighestPrice());
						entity.setSumLowestPrice(entity.getTotalLowestPrice());
						entity.setTotalHighestTaxesPrice(0.0);
						entity.setTotalLowestTaxesPrice(0.0);
					}else{
						logger.warn(String.format("截取字符串[%s]不能转化为价格", priceStr));
					}
				}
			}
			if(!doc.select("tr.layover").isEmpty()){//非直达
				for(int i=0;i<=doc.select("tr.layover").size();i++){//有n+1个中转
					String trAreaName=RegHtmlUtil.regStr(doc.select("tr.first").get(i).text(), "(.*?)–").replaceAll("\\s", "");
					String trFlightNo=getCarrierKey(trAreaName)+RegHtmlUtil.regStr(doc.select("tr.first").get(i).text(), getREGEX_FLIGHTNO()+".*?(\\d.*?)\\s").replaceAll("\\s", "");
					String trStart=RegHtmlUtil.regStr(doc.select("tr.takeoff").get(i).select("td.time").first().text(),"(\\d{1,2}:\\d{1,2})").replaceAll("\\s", "");
					String trStartDate=doc.select("tr.takeoff").get(i).select("td.date").first().text();
					trStart=this.tranDateTime(null,trStart,trStartDate);
					trStart=trStart.length()==4?"0"+trStart:start;
					String trEnd=RegHtmlUtil.regStr(doc.select("tr.landing").get(i).select("td.time").first().text(),"(\\d{1,2}:\\d{1,2})").replaceAll("\\s", "");
					String trEndDate=doc.select("tr.landing").get(i).select("td.date").first().text();
					trEnd=this.tranDateTime(null,trEnd,trEndDate);
					trEnd=trEnd.length()==4?"0"+trEnd:end;
					String startAirPort=RegHtmlUtil.regStr(doc.select("tr.takeoff").get(i).select("td.airport").first().text(), "(\\w{3})");
					String endAirPort=RegHtmlUtil.regStr(doc.select("tr.landing").get(i).select("td.airport").first().text(), "(\\w{3})");
					TransitEntity trEntity=PlaneInfoEntityBuilder.buildTransitEntity(
							trFlightNo, trFlightNo, getCarrierKey(trAreaName), trAreaName, trAreaName, 
							startAirPort, null, endAirPort, null, null);
					//modify 14/06/26 设置时间
					try{
						trEntity.setStartTime(CrawlAdapterFactory.getFlightTime(taskQueue.getReturnGrabDate(), trStart));
						trEntity.setEndTime(CrawlAdapterFactory.getFlightTime(taskQueue.getReturnGrabDate(), trEnd));
					}catch(Exception e){
						logger.info("单程设置中转时间失败");
						logger.info(e);
					}
					entity.getTransits().add(trEntity);
				}
			}
			//增加税收字段(Kayak只有打包价)
			entity.setSumHighestPrice(entity.getTotalHighestPrice());
			entity.setSumLowestPrice(entity.getTotalLowestPrice());
			entity.setTotalHighestTaxesPrice(0.0);
			entity.setTotalLowestTaxesPrice(0.0);
			
			entitys.add(entity);
		}
		return Arrays.asList(entitys.toArray());
	}

	private String getPrice(String text){
		return text.replaceAll("[^\\d\\.]", "");
	}
	
	//往返
	@SuppressWarnings("unchecked")
	public List<Object> paraseoRound(Object fetchObject) throws Exception {
		List<String> htmls=(List<String>)fetchObject;
		List<DoublePlaneInfoEntity> entitys=new ArrayList<DoublePlaneInfoEntity>();
		for(String html:htmls){
			/**
			 ***********************去程**********************
			 */	
			Document doc=Jsoup.parse("<html><body>"+html+"</body></html>");
			String gHtml=RegHtmlUtil.regStr(html, "(<tr.*?class=\"header.*)<tr.*?class=\"header\"");//	去程信息
			Document gd=Jsoup.parse("<html><body><table>"+gHtml+"</table></body></html>");
			String gAirName=RegHtmlUtil.regStr(gd.select("tr.first").first().text(), "(.*?)–").replaceAll("\\s", "");
			String gFlightNo=getCarrierKey(gAirName)+RegHtmlUtil.regStr(gd.select("tr.first").first().text(), getREGEX_FLIGHTNO()+".*?(\\d.*?)\\s").replaceAll("\\s", "");
			String totalFlighTime=gd.select("td.duration").first().text();//RegHtmlUtil.regStr(gd.select("td.duration").first().text(), "(\\d{1,2}+h+\\s+\\d{1,2}+m)").replaceAll("\\s", "");//总飞行时间XXh XXm
//			String gStart=RegHtmlUtil.regStr(gd.select("td.time").first().text(),"(\\d{1,2}:\\d{1,2}+[a-zA-Z]?+$)").replaceAll("\\s", "");
			String gStart=gd.select("td.time").first().text();
			String gStartDate=gd.select("td.date").first().text();
			gStart=this.tranDateTime(taskQueue.getFlightDate(),gStart,gStartDate);
//			gStart=gStart.length()==4?"0"+gStart:gStart;
//			String gEnd=RegHtmlUtil.regStr(gd.select("td.time").last().text(),"(\\d{1,2}:\\d{1,2}+[a-zA-Z]?+$)").replaceAll("\\s", "");
			String gEnd=gd.select("td.time").last().text();
			String gEndDate=gd.select("td.date").last().text();
			gEnd=this.tranDateTime(taskQueue.getFlightDate(),gEnd,gEndDate);
//			gEnd=gEnd.length()==4?"0"+gEnd:gEnd;
			DoublePlaneInfoEntity entity=PlaneInfoEntityBuilder.buildPlaneInfo(
					taskQueue, getCarrierKey(gAirName), gAirName, gAirName,null,null, gFlightNo, null, null, null, null, DoublePlaneInfoEntity.class);
			entity.setStartTime(DateUtil.StringToDate("yyyy-MM-dd HH:mm", gStart));
			entity.setEndTime(DateUtil.StringToDate("yyyy-MM-dd HH:mm", gEnd));
			entity.setFlightDuration(stayTime(totalFlighTime));//飞行总时间
			entity.setCurrency("CNH");
			String stopTime=null;
			if(!gd.select("tr.layover").isEmpty()){//去程非直达
				for(int i=0;i<=gd.select("tr.layover").size();i++){
					String gTrAreaName=RegHtmlUtil.regStr(gd.select("tr.first").get(i).text(), "(.*?)–").replaceAll("\\s", "");
					String gTrFlightNo=getCarrierKey(gTrAreaName)+RegHtmlUtil.regStr(gd.select("tr.first").get(i).text(), getREGEX_FLIGHTNO()+".*?(\\d.*?)\\s").replaceAll("\\s", "");
//					String gTrStart=RegHtmlUtil.regStr(gd.select("tr.takeoff").get(i).select("td.time").first().text(),"(\\d{1,2}:\\d{1,2}+[a-zA-Z]?+$)").replaceAll("\\s", "");
					String gTrStart=gd.select("tr.takeoff").get(i).select("td.time").first().text();
					String gTrStartDate=gd.select("tr.takeoff").get(i).select("td.date").first().text();
					gTrStart=this.tranDateTime(taskQueue.getFlightDate(),gTrStart,gTrStartDate);
//					gTrStart=gTrStart.length()==4?"0"+gTrStart:gTrStart;
//					String gTrEnd=RegHtmlUtil.regStr(gd.select("tr.landing").get(i).select("td.time").first().text(),"(\\d{1,2}:\\d{1,2}+[a-zA-Z]?+$)").replaceAll("\\s", "");
					String gTrEnd=gd.select("tr.landing").get(i).select("td.time").first().text();
					String gTrEndDate=gd.select("tr.landing").get(i).select("td.date").first().text();
					gTrEnd=this.tranDateTime(taskQueue.getFlightDate(),gTrEnd,gTrEndDate);
//					gTrEnd=gTrEnd.length()==4?"0"+gTrEnd:gTrEnd;
					String gStartAirPort=RegHtmlUtil.regStr(gd.select("tr.takeoff").get(i).select("td.airport").first().text(), "(\\w{3})");
					String gEndAirPort=RegHtmlUtil.regStr(gd.select("tr.landing").get(i).select("td.airport").first().text(), "(\\w{3})");
					String flightInfo=gd.getElementsByTag("tr").last().text();//中转信息
					String flightTyle=null;
					flightTyle=RegHtmlUtil.regStr(flightInfo,"(\\w*\\s*\\w*\\-\\w*\\s*?)");//flightInfo.split("\\|")[2].split("\\(")[0].replaceAll(" ", "");//飞机类型
					if( flightTyle ==null || flightTyle.equals("")){
						flightTyle=RegHtmlUtil.regStr(flightInfo,"([a-zA-z]+\\s+[a-zA-z0-9]*)");
					}
					if(i!=gd.select("tr.layover").size()){//中转只有一个停留城市
						stopTime=gd.select("td.duration div").text();//RegHtmlUtil.regStr(flightInfo.replaceAll(" ", ""),"(\\d{1,2}+h\\d{1,2}+m)");
					}
				//	System.out.println("flightTyle=="+flightTyle);// Boeing 777-300ER (宽体喷气式飞机)()
					if(i==0){
						entity.setFlightType(flightTyle);
					}
					TransitEntity trEntity=PlaneInfoEntityBuilder.buildTransitEntity(
							gTrFlightNo, gTrFlightNo, getCarrierKey(gTrAreaName), gTrAreaName, gTrAreaName, 
							gStartAirPort, null, gEndAirPort,null, flightTyle,TransitEntity.class);
					trEntity.setStayTime(stayTime(stopTime));//停留时间：毫秒
					try{
						trEntity.setStartTime(DateUtil.StringToDate("yyyy-MM-dd HH:mm", gTrStart));
						trEntity.setEndTime(DateUtil.StringToDate("yyyy-MM-dd HH:mm", gTrEnd));
//						trEntity.setStartTime(CrawlAdapterFactory.getFlightTime(taskQueue.getFlightDate(), gTrStart));
//						trEntity.setEndTime(CrawlAdapterFactory.getFlightTime(taskQueue.getFlightDate(), gTrEnd));
					}catch(Exception e){
						logger.info("单程设置往返中转时间失败");
						logger.info(e);
					}
					entity.getTransits().add(trEntity);
				}
			}
			//对去程进行合并，按照业务需求，会有一去多回的情况
			entity=findSameInResult(entity,entitys);
			if(!entitys.contains(entity)){
				entitys.add(entity);//放入结果集
			}
			/**
			 ***********************回程**********************
			 */			
			String rHtml=RegHtmlUtil.regStr(html, "<tr.*?class=\"header.*(<tr.*?class=\"header\".*?)<tbody");//回程信息
			Document rd=Jsoup.parse("<html><body><table>"+rHtml+"</table></body></html>");
			String rAirName=RegHtmlUtil.regStr(rd.select("tr.first").first().text(), "(.*?)–").replaceAll("\\s", "");
			String rFlightNo=getCarrierKey(rAirName)+RegHtmlUtil.regStr(rd.select("tr.first").first().text(), getREGEX_FLIGHTNO()+".*?(\\d.*?)\\s").replaceAll("\\s", "");
			String totalFlighTimeR=gd.select("td.duration").first().text();//RegHtmlUtil.regStr(gd.select("td.duration").first().text(), "(\\d{1,2}+h+\\s+\\d{1,2}+m)").replaceAll("\\s", "");//总飞行时间XXh XXm
//			String rStart=RegHtmlUtil.regStr(rd.select("td.time").first().text(),"(\\d{1,2}:\\d{1,2}+[a-zA-Z]?+$)").replaceAll("\\s", "");
			String rStart=rd.select("td.time").first().text();
			String rStartDate=rd.select("td.date").first().text();
			rStart=this.tranDateTime(taskQueue.getReturnGrabDate(),rStart,rStartDate);
//			rStart=rStart.length()==4?"0"+rStart:rStart;
//			String rEnd=RegHtmlUtil.regStr(rd.select("td.time").last().text(),"(\\d{1,2}:\\d{1,2}+[a-zA-Z]?+$)").replaceAll("\\s", "");
			String rEnd=rd.select("td.time").last().text();
			String rEndDate=rd.select("td.date").last().text();
			rEnd=this.tranDateTime(taskQueue.getReturnGrabDate(),rEnd,rEndDate);
//			rEnd=rEnd.length()==4?"0"+rEnd:rEnd;
			
			ReturnDoublePlaneInfoEntity rEntity=PlaneInfoEntityBuilder.buildPlaneInfo(
					taskQueue, getCarrierKey(rAirName), rAirName, rAirName, null,null, rFlightNo, null, null, null, null, ReturnDoublePlaneInfoEntity.class);
			
			rEntity.setStartTime(DateUtil.StringToDate("yyyy-MM-dd HH:mm", rStart));
			rEntity.setEndTime(DateUtil.StringToDate("yyyy-MM-dd HH:mm", rEnd));
			rEntity.setFlightDuration(stayTime(totalFlighTimeR));//飞行总时间
			rEntity.setCurrency("CNH");
			entity.getReturnPlaneInfos().add(rEntity);//回程放入
			String stopTimeR=null;
			if(!rd.select("tr.layover").isEmpty()){//回程非直达
				for(int i=0;i<=rd.select("tr.layover").size();i++){
					String rTrAreaName=RegHtmlUtil.regStr(rd.select("tr.first").get(i).text(), "(.*?)–").replaceAll("\\s", "");
					String rTrFlightNo=getCarrierKey(rTrAreaName)+RegHtmlUtil.regStr(rd.select("tr.first").get(i).text(), getREGEX_FLIGHTNO()+".*?(\\d.*?)\\s").replaceAll("\\s", "");
//					String rTrStart=RegHtmlUtil.regStr(rd.select("tr.takeoff").get(i).select("td.time").first().text(),"(\\d{1,2}:\\d{1,2}+[a-zA-Z]?+$)").replaceAll("\\s", "");
					String rTrStart=rd.select("tr.takeoff").get(i).select("td.time").first().text();
					String rTrStartDate=rd.select("tr.takeoff").get(i).select("td.date").first().text();
					rTrStart=this.tranDateTime(taskQueue.getReturnGrabDate(),rTrStart,rTrStartDate);
//					rTrStart=rTrStart.length()==4?"0"+rTrStart:rTrStart;
//					String rTrEnd=RegHtmlUtil.regStr(rd.select("tr.landing").get(i).select("td.time").first().text(),"(\\d{1,2}:\\d{1,2}+[a-zA-Z]?+$)").replaceAll("\\s", "");
					String rTrEnd=rd.select("tr.landing").get(i).select("td.time").first().text();
					String rTrEndDate=rd.select("tr.landing").get(i).select("td.date").first().text();
					rTrEnd=this.tranDateTime(taskQueue.getReturnGrabDate(),rTrEnd,rTrEndDate);
//					rTrEnd=rTrEnd.length()==4?"0"+rTrEnd:rTrEnd;
					String rStartAirPort=RegHtmlUtil.regStr(rd.select("tr.takeoff").get(i).select("td.airport").first().text(), "(\\w{3})");
					String rEndAirPort=RegHtmlUtil.regStr(rd.select("tr.landing").get(i).select("td.airport").first().text(), "(\\w{3})");
					String flightInfoR=rd.getElementsByTag("tr").select("td.extra").get(i).text();//中转信息
					String flightTyleR=null;
					flightTyleR=RegHtmlUtil.regStr(flightInfoR,"(\\w*\\s*\\w*\\-\\w*\\s*?)");//flightInfoR.split("\\|")[2].split("\\(")[0].replaceAll(" ", "");//飞机类型
					if(flightTyleR ==null || flightTyleR.equals("")){
						flightTyleR=RegHtmlUtil.regStr(flightInfoR,"([a-zA-z]+\\s+[a-zA-z0-9]*)");
					}
				//	System.out.println("flightTyleR=="+flightTyleR);//舱  |  Boeing 777-300ER (宽体喷气式飞机)  | 
					if(i!=rd.select("tr.layover").size()){//(\\w*\\s*\\w*\\-\\w*\\s*?)
						stopTimeR=rd.select("td.duration div").text();//RegHtmlUtil.regStr(flightInfoR.replaceAll(" ", ""),"(\\d{1,2}+h\\d{1,2}+m)");
					}
					if(i==0){
						rEntity.setFlightType(flightTyleR);
					}
					ReturnTransitEntity rTr=PlaneInfoEntityBuilder.buildTransitEntity(
							rTrFlightNo, rTrFlightNo, getCarrierKey(rTrAreaName), rTrAreaName, rTrAreaName, 
							rStartAirPort, null, rEndAirPort, null, null,ReturnTransitEntity.class);
					try{
						rTr.setStartTime(DateUtil.StringToDate("yyyy-MM-dd HH:mm", rTrStart));
						rTr.setEndTime(DateUtil.StringToDate("yyyy-MM-dd HH:mm", rTrEnd));
//						rTr.setStartTime(CrawlAdapterFactory.getFlightTime(taskQueue.getFlightDate(), rTrStart));
//						rTr.setEndTime(CrawlAdapterFactory.getFlightTime(taskQueue.getFlightDate(), rTrEnd));
					}catch(Exception e){
						logger.info("单程设置往返回程中转时间失败");
						logger.info(e);
					}
					rTr.setStayTime(stayTime(stopTimeR));//停留时间
					rTr.setFlightType(flightTyleR);
					rEntity.getReturnTransits().add(rTr);
				}
			}
			
			//仓位价格信息，放入回程实体,打包价
			for(Element div:doc.select("div.fareInformation")){//预订网站
				String cabinName=div.attr("data-cabin");
				for(Element tb:div.select("tbody")){
				//	String agentName=tb.select(".image").first().attr("title");
					//String cabinCode=tb.select("td.fareCodes").text();//中文版没有了舱位
					String priceStr=getPrice(tb.select("td.total").text());
					//String priceStr=tb.select("td.total").text().replaceAll("\\D", "");
					logger.info(String.format("[%s]解析为[%s]", tb.select("td.total").text(),priceStr));
					if(NumberUtils.isNumber(priceStr)){
						Double price=Double.valueOf(priceStr);
						ReturnCabinEntity rCabinE=PlaneInfoEntityBuilder.buildCabinInfo(
								cabinName, null, null, priceStr, null, null, null, ReturnCabinEntity.class);
						
						//税收信息
						rCabinE.setTaxesPrice(0.0);
						rCabinE.setOriginalPrice(rCabinE.getPrice());
						
						rEntity.getReturnCabins().add(rCabinE);
						ReturnAgentEntity rAgent=new ReturnAgentEntity();
						rAgent.setPrice(price);
						rEntity.getReturnAgents().add(rAgent);
						
						if(rEntity.getSumLowestPrice()==null ||rEntity.getSumLowestPrice()>price){
							rEntity.setSumLowestPrice(price);
						}
						if(rEntity.getSumHighestPrice()==null || price>rEntity.getSumHighestPrice()){
							rEntity.setSumHighestPrice(price);
						}
						rEntity.setTotalLowestPrice(0.0);
						rEntity.setTotalHighestPrice(0.0);
						
						//去程里面放最低和最高  Kayak只有打包价
						if(entity.getSumLowestPrice()==null || price<entity.getSumLowestPrice()){
							entity.setSumLowestPrice(price);
						}
						if(entity.getSumHighestPrice()==null || price>entity.getSumHighestPrice()){
							entity.setSumHighestPrice(price);
						}
						//增加税收字段(Kayak只有打包价)
						entity.setTotalLowestPrice(0.0);
						entity.setTotalHighestPrice(0.0);
						entity.setTotalHighestTaxesPrice(0.0);
						entity.setTotalLowestTaxesPrice(0.0);
					}else{
						logger.warn(String.format("截取字符串[%s]不能转化为价格", priceStr));
					}
				}
			}
			
		}
		return Arrays.asList(entitys.toArray()); 
	}
	
	private DoublePlaneInfoEntity findSameInResult(DoublePlaneInfoEntity doubleEntity,List<DoublePlaneInfoEntity> lists){
		if(doubleEntity==null || lists==null || lists.isEmpty()){
			return doubleEntity;
		}
		for(DoublePlaneInfoEntity l:lists){
			if(getFlightNos(doubleEntity).equals(l)){
				return l;
			}
		}
		return doubleEntity;
	}
	
	private String getFlightNos(DoublePlaneInfoEntity d){
		String flightNos = null;
		if(d.getTransits()!=null && !d.getTransits().isEmpty()){
			for(TransitEntity tr:d.getTransits()){
				flightNos=flightNos==null?tr.getFlightNo():(flightNos+tr.getFlightNo());
			}
		}else{
			flightNos=d.getFlightNo();
		}
		return flightNos;
	}
	
	
	private String getCarrierKey(String airName){
		airName=airName.replaceAll("\\s", "");
		if("AirChina".equalsIgnoreCase(airName)){
			return "CA";
		}else if("HainanAirlines".equalsIgnoreCase(airName)){
			return "HU";
		}else if("ChinaSouthern".equalsIgnoreCase(airName)){
			return "CZ";
		}else if("ShandongAirlines".equalsIgnoreCase(airName)){
			return "SC";
		}else if("ChinaEasternAir".equalsIgnoreCase(airName)){
			return "MU";
		}else if("Emirates".equalsIgnoreCase(airName)){
			return "EK";
		}else if("ShenzhenAirlines".equalsIgnoreCase(airName)){
			return "ZH";
		}else if("XiamenAirlines".equalsIgnoreCase(airName)){
			return "MF";
		}else if("SichuanAirlines".equalsIgnoreCase(airName)){
			return "3U";
		}else if("Emirates".equalsIgnoreCase(airName)){
			return "EK";
		}else if("United".equalsIgnoreCase(airName)){
			return "UA";
		}else if("Delta".equalsIgnoreCase(airName)){
			return "DL";
		}else if("AmericanAirlines".equalsIgnoreCase(airName)){
			return "AA";
		}else if("AirCanada".equalsIgnoreCase(airName)){
			return "AC";
		}else if("HawaiianAirlines".equalsIgnoreCase(airName)){
			return "HA";
		}else if("AsianaAirlines".equalsIgnoreCase(airName)){
			return "OZ";
		}else if("KoreanAir".equalsIgnoreCase(airName)){
			return "KE";
		}else if("ANA".equalsIgnoreCase(airName)){
			return "NH";
		}else if("JapanAirlines".equalsIgnoreCase(airName)){
			return "JL";
		}else if("CathayPacific".equalsIgnoreCase(airName)){
			return "CX";
		}else if("HongKongAirlines".equalsIgnoreCase(airName)){
			return "HX";
		}else if("Aeroflot".equalsIgnoreCase(airName)){
			return "SU";
		}else if("Lufthansa".equalsIgnoreCase(airName)){
			return "LH";
		}else if("AirFrance".equalsIgnoreCase(airName)){
			return "AF";
		}else{
			logger.warn("未知航空公司代码，请收录:"+airName);
			return "UN";
		}
		
	}
	
	private String getCabinType(String cabinName){
		cabinName=cabinName.replaceAll("\\s", "");
		if("Economy".equals(cabinName)){
			return "经济舱";
		}else if("First".equals(cabinName)){
			return "头等舱";
		}else if("Premium".equals(cabinName)){
			return "豪華經濟";
		}else{
			return cabinName;
		}
	}
	
	
	//重设一个日期
	public String tranDateTime(String dateStr,String str,String dateStrs){//str ="Fri 10:10a";
		//日期转换
		String weekStr=null ,hm="",finalDate="",y="",m="",d="";
		int flightDay=0;
		if(dateStr !=null && dateStrs !=null ){
			weekStr=todayOFweek(DateUtil.StringToDate("yyyy-MM-dd", dateStr));
			
			int curentDay=dateMap.get(weekStr);
			
			if(dateStrs !=null && !dateStrs.equals("")){
				String compare=dateStrs.split(" ")[1];//网站
				flightDay=dateMap.get(compare);
			}
			
			/*if(str !=null && !str.equals("")){
				String compare=str.split(" ")[0];//网站
				flightDay=dateMap.get(compare);
			}*/
			if(flightDay >0 && flightDay-curentDay !=0){
				
				finalDate=DateUtil.formatDay(DateUtils.addDays(DateUtil.StringToDate("yyyy-MM-dd",dateStr), flightDay-curentDay),"yyyy-MM-dd");
				//DateUtil.getDateAfter(DateUtil.StringToDate("yyyy-MM-dd",dateStr),flightDay-curentDay);
			}else{
				finalDate=dateStr;
			}

		}
		
		if(str !=null && !str.equals("")){
			hm=str.replaceAll("[\u4e00-\u9fa5]", "").replaceAll(" ", "");
			/*String ap=str.split(" ")[1].replaceAll("[^(a-zA-Z)]","");
			if(ap.equalsIgnoreCase("p") && !str.split(" ")[1].split("\\:")[0].equalsIgnoreCase("12")){
				int h=Integer.parseInt(str.split(" ")[1].split("\\:")[0])+12;
				hm=h+":"+str.split(" ")[1].split("\\:")[1].replaceAll("[^(0-9)]", "");
			}else{
				hm=str.split(" ")[1].replaceAll("[(a-zA-Z)]*", "");
			}*/
		}
		return finalDate+" "+hm.replaceAll(" ", "");
	}
	
	//处理停留时间long
	public Long stayTime(String str){
		//时间转换
		//英文网站
		/*if(str !=null && !str.equals("")){
			Long h=Long.parseLong(RegHtmlUtil.regStr(str,"(\\d{1,2}+h)").replaceAll("[^(0-9)]", ""));
			Long m=Long.parseLong(RegHtmlUtil.regStr(str,"(\\d{1,2}+m)").replaceAll("[^(0-9)]", ""));
			Long stayTimes=0l;
			stayTimes=h*60+m;
			return stayTimes*60*1000;//毫秒
		}*/
		//中文网站 
		if(str !=null && !str.equals("")){
			Long h=Long.parseLong(str.split("小时")[0]);
			Long m=Long.parseLong(str.split("小时")[1].replaceAll("[^(0-9)]", ""));
			Long stayTimes=0l;
			stayTimes=h*60+m;
			return stayTimes*60*1000;//毫秒
		}
		return null;
	}


	public String todayOFweek(Date date){
		if(date !=null){
			//英文
//			String[] weekDays = {"Sun", "Mon", "Tue", "Wed", "Thr", "Fri", "Sat"};
			String[] weekDays = {"星期日", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六"};
	        Calendar cal = Calendar.getInstance();
	        cal.setTime(date);
	        int w = cal.get(Calendar.DAY_OF_WEEK) - 1;
	        if (w < 0) w = 0;
	        return weekDays[w];
		}
		return null;
	}
	
	
	@Override
	public String getUrl() throws Exception {
		return null;
	}

	@Override
	public boolean validateFetch(Object fetchObject) throws Exception {
		return true;
	}
	
	  public static void main(String[] args) throws Exception {
	        TaskModel taskModel = new TaskModel();
	        taskModel.setFromCity("PEK"); 
	        taskModel.setToCity("SEA");
	        taskModel.setFromCityName("北京");
	        taskModel.setToCityName("旧金山");
	        taskModel.setFlightDate("2015-01-10");
	        taskModel.setIsReturn(1);
	        taskModel.setIsInternational(1);
	        taskModel.setReturnGrabDate("2015-02-11");
	        KayakAdapter adapter = new KayakAdapter(taskModel);
	        Object obj = adapter.fetch(null);
			List<Object> objList = adapter.paraseToVo(obj);
			for(Object o : objList){
				SinglePlaneInfoEntity entity = (SinglePlaneInfoEntity)o;
				System.out.println("entity:" + entity.getFlightNo() + ",higePrice" + entity.getTotalHighestPrice()
						+ ",lowerPrice:" + entity.getTotalLowestPrice());
				for(CabinEntity cabin : entity.getCabins()){
					System.out.println("\tcabin:" + cabin.getCabinType() + ",price:" + cabin.getPrice());
				}
				for(TransitEntity transEntity : entity.getTransits()){
					System.out.println("\ttrans:" + transEntity.getActuallyFlightNo());
				}
			}
	    }

	@Override
	public boolean switchProxyip() {
		// TODO Auto-generated method stub
		return false;
	}

}
